{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.layers import Dense, Input, Flatten\n",
    "from keras.layers import GlobalMaxPool1D, Bidirectional, Convolution1D, Embedding, BatchNormalization,MaxPooling1D, Dropout, LSTM\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "INPUT_PATH = '../input/'\n",
    "CACHE_PATH = '../cache/'\n",
    "OUTPUT_PATH ='../output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = np.load(CACHE_PATH + 'data.npz')\n",
    "X_train = data['X_train']\n",
    "y_train = data['y_train']\n",
    "X_val = data['X_val']\n",
    "y_val = data['y_val']\n",
    "X_test = data['X_test']\n",
    "embedding_matrix = np.load(CACHE_PATH + 'embedding_matrix.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_FEATURES = 20000\n",
    "MAX_SEQUENCE_LENGTH = 100\n",
    "EMBEDDING_DIM = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(MAX_FEATURES,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        \"\"\"\n",
    "        Keras Layer that implements an Attention mechanism for temporal data.\n",
    "        Supports Masking.\n",
    "        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n",
    "        # Input shape\n",
    "            3D tensor with shape: `(samples, steps, features)`.\n",
    "        # Output shape\n",
    "            2D tensor with shape: `(samples, features)`.\n",
    "        :param kwargs:\n",
    "        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "        The dimensions are inferred based on the output shape of the RNN.\n",
    "        Example:\n",
    "            model.add(LSTM(64, return_sequences=True))\n",
    "            model.add(Attention())\n",
    "        \"\"\"\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        # eij = K.dot(x, self.W) TF backend doesn't support it\n",
    "\n",
    "        # features_dim = self.W.shape[0]\n",
    "        # step_dim = x._keras_shape[1]\n",
    "\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        # apply mask after the exp. will be re-normalized next\n",
    "        if mask is not None:\n",
    "            # Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # in some cases especially in the early stages of training the sum may be almost zero\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "    #print weigthted_input.shape\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        #return input_shape[0], input_shape[-1]\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_attention_model():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(inp)\n",
    "    x = Bidirectional(LSTM(250, dropout=0.25, recurrent_dropout=0.25, return_sequences=True))(embedded_sequences)\n",
    "    merged = Attention(MAX_SEQUENCE_LENGTH)(x)\n",
    "    merged = Dense(256, activation='relu')(merged)\n",
    "    merged = Dropout(0.25)(merged)\n",
    "    merged = BatchNormalization()(merged)\n",
    "    preds = Dense(1, activation='linear')(merged)\n",
    "    model = Model(inputs=inp, outputs=preds)\n",
    "    model.compile(loss='mse',optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_attention_model(model):\n",
    "    model_path = CACHE_PATH + \"attention_weights_best.hdf5\"\n",
    "    early = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5)\n",
    "    checkpoint = ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "    callbacks_list = [checkpoint, early]\n",
    "    model.fit(X_train, y_train, batch_size=128, epochs=100, validation_data=(X_val, y_val), callbacks=callbacks_list)\n",
    "    model.load_weights(model_path)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_attention_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model,to_file='Atttention.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 100, 256)          5120000   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 100, 500)          1014000   \n",
      "_________________________________________________________________\n",
      "attention_1 (Attention)      (None, 500)               600       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               128256    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 6,264,137\n",
      "Trainable params: 1,143,625\n",
      "Non-trainable params: 5,120,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 209000 samples, validate on 11000 samples\n",
      "Epoch 1/100\n",
      "209000/209000 [==============================] - 324s 2ms/step - loss: 1.2565 - val_loss: 0.6416\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.64161, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 2/100\n",
      "209000/209000 [==============================] - 325s 2ms/step - loss: 0.4446 - val_loss: 0.5350\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.64161 to 0.53503, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 3/100\n",
      "209000/209000 [==============================] - 323s 2ms/step - loss: 0.4313 - val_loss: 0.4487\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.53503 to 0.44874, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 4/100\n",
      "209000/209000 [==============================] - 322s 2ms/step - loss: 0.4226 - val_loss: 0.4299\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.44874 to 0.42988, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 5/100\n",
      "209000/209000 [==============================] - 326s 2ms/step - loss: 0.4153 - val_loss: 0.4652\n",
      "\n",
      "Epoch 00005: val_loss did not improve\n",
      "Epoch 6/100\n",
      "209000/209000 [==============================] - 328s 2ms/step - loss: 0.4065 - val_loss: 0.4179\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.42988 to 0.41793, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 7/100\n",
      "209000/209000 [==============================] - 324s 2ms/step - loss: 0.3994 - val_loss: 0.4162\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.41793 to 0.41618, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 8/100\n",
      "209000/209000 [==============================] - 327s 2ms/step - loss: 0.3902 - val_loss: 0.4261\n",
      "\n",
      "Epoch 00008: val_loss did not improve\n",
      "Epoch 9/100\n",
      "209000/209000 [==============================] - 324s 2ms/step - loss: 0.3825 - val_loss: 0.4326\n",
      "\n",
      "Epoch 00009: val_loss did not improve\n",
      "Epoch 10/100\n",
      "209000/209000 [==============================] - 323s 2ms/step - loss: 0.3733 - val_loss: 0.4141\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.41618 to 0.41411, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 11/100\n",
      "209000/209000 [==============================] - 324s 2ms/step - loss: 0.3674 - val_loss: 0.4122\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.41411 to 0.41223, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 12/100\n",
      "209000/209000 [==============================] - 322s 2ms/step - loss: 0.3615 - val_loss: 0.4103\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.41223 to 0.41029, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 13/100\n",
      "209000/209000 [==============================] - 323s 2ms/step - loss: 0.3545 - val_loss: 0.4145\n",
      "\n",
      "Epoch 00013: val_loss did not improve\n",
      "Epoch 14/100\n",
      "209000/209000 [==============================] - 324s 2ms/step - loss: 0.3492 - val_loss: 0.4246\n",
      "\n",
      "Epoch 00014: val_loss did not improve\n",
      "Epoch 15/100\n",
      "209000/209000 [==============================] - 324s 2ms/step - loss: 0.3435 - val_loss: 0.4094\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.41029 to 0.40941, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 16/100\n",
      "209000/209000 [==============================] - 330s 2ms/step - loss: 0.3404 - val_loss: 0.4185\n",
      "\n",
      "Epoch 00016: val_loss did not improve\n",
      "Epoch 17/100\n",
      "209000/209000 [==============================] - 325s 2ms/step - loss: 0.3356 - val_loss: 0.4286\n",
      "\n",
      "Epoch 00017: val_loss did not improve\n",
      "Epoch 18/100\n",
      "209000/209000 [==============================] - 334s 2ms/step - loss: 0.3310 - val_loss: 0.4317\n",
      "\n",
      "Epoch 00018: val_loss did not improve\n",
      "Epoch 19/100\n",
      "209000/209000 [==============================] - 325s 2ms/step - loss: 0.3259 - val_loss: 0.4199\n",
      "\n",
      "Epoch 00019: val_loss did not improve\n",
      "Epoch 20/100\n",
      "209000/209000 [==============================] - 324s 2ms/step - loss: 0.3242 - val_loss: 0.4287\n",
      "\n",
      "Epoch 00020: val_loss did not improve\n"
     ]
    }
   ],
   "source": [
    "model = train_attention_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 26s 527us/step\n"
     ]
    }
   ],
   "source": [
    "y_test = model.predict(X_test,batch_size=128,verbose=1)\n",
    "y_test[y_test < 1] = 1\n",
    "y_test[y_test > 4.7] = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv(INPUT_PATH + 'sample.csv',header=None,names=['Id','Score'])\n",
    "sub['Score'] = y_test\n",
    "sub.to_csv(OUTPUT_PATH + 'attention.csv',index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d1c0b520-43c2-3060-843f-711422be08e7</td>\n",
       "      <td>3.089528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fed809ea-6c05-3cb5-864f-0b10199f38cf</td>\n",
       "      <td>4.419052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62880c2f-19ad-367d-ba6b-285984fd2e1c</td>\n",
       "      <td>4.084131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7a35d3c4-f5ff-384e-b0f9-032f3be7d81b</td>\n",
       "      <td>4.546053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>64f0cdd4-26f9-3034-b109-eac59a1f4f30</td>\n",
       "      <td>4.691096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8fa29e8b-49e4-3d65-8dc1-9402c61dabd9</td>\n",
       "      <td>4.645653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>512fc844-4f4f-3cdc-93a7-2f209937420a</td>\n",
       "      <td>3.875635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>eb68cf16-ae60-3a1e-b648-ad92cc12c9c4</td>\n",
       "      <td>4.045265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>89e8804e-84d1-37ce-8af0-ae3cd38f7b06</td>\n",
       "      <td>4.462513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>6b6abc03-ae74-35a1-9b4f-ec4525ab3b72</td>\n",
       "      <td>4.522406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>a96151dc-0d88-3a37-bd19-3dd426704414</td>\n",
       "      <td>3.894797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>64a2cd30-8e29-3df4-b580-5b5d82103c31</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>33a3b298-5e75-3b39-b445-d7deafc5704a</td>\n",
       "      <td>4.614712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>26ea0a19-d13b-3ffe-8c60-0f34a4a0a634</td>\n",
       "      <td>4.277895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>81915b19-7cc0-31c1-b271-c005a5d01bb5</td>\n",
       "      <td>4.681723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>82a3142b-edb1-31e0-977f-6cdb96e1dbf7</td>\n",
       "      <td>4.495889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>01cca46c-88cd-3c87-9583-f65999bd35c4</td>\n",
       "      <td>4.636938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>b45685cd-641c-3ed8-bfb9-8ffdb5b29d1b</td>\n",
       "      <td>3.842805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2b4e3e0b-1f54-3fb7-95e5-e50c638a5435</td>\n",
       "      <td>2.444538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>9e20d005-b766-3b53-ade8-2cbf226bd318</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>4df2cbb1-6994-3998-8cb6-21e1bd00e041</td>\n",
       "      <td>4.526083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>3539405d-e389-3ec0-923c-ddaeecd3803c</td>\n",
       "      <td>3.891205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>284baf9d-a66e-3717-9735-e7bfe3c90e5b</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>4ea8f7a1-4219-3731-a960-888b0061c34b</td>\n",
       "      <td>4.500006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>99cea5e2-95d1-3b1e-99c4-5f989a7e498a</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>5cceae79-a341-375e-863c-23bf1a98a4ee</td>\n",
       "      <td>4.330118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>106b3b7d-4b95-3006-bb3a-24e577c44967</td>\n",
       "      <td>4.547858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>89cd390d-4bde-3421-8806-199f98d74cb1</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>3e1b59a4-d5b6-30f3-b6ea-8d70b04fe445</td>\n",
       "      <td>4.452993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>5d98909a-41f1-3ec4-9ea1-145d93e8530f</td>\n",
       "      <td>3.861439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49970</th>\n",
       "      <td>75e3f467-ee98-3dcc-812f-72f7ab6bf80c</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49971</th>\n",
       "      <td>78039063-aa5d-34ab-a1a6-00d49ff6a16f</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49972</th>\n",
       "      <td>f8aec5d7-b10c-3005-891a-3e353765cea0</td>\n",
       "      <td>3.583248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49973</th>\n",
       "      <td>a056a723-7270-3bf3-ba48-b590d2367f33</td>\n",
       "      <td>4.650035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49974</th>\n",
       "      <td>fb7dfd18-b884-3bb0-88da-f6a91aa2908e</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49975</th>\n",
       "      <td>61c44262-2ba0-3764-b2fa-034afa11ba63</td>\n",
       "      <td>3.804826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49976</th>\n",
       "      <td>168ffccc-97f2-3292-89f1-b5e69e2b0342</td>\n",
       "      <td>4.511363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49977</th>\n",
       "      <td>11d09de4-6b44-384e-b2ab-e09c70ccb7d2</td>\n",
       "      <td>4.446552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49978</th>\n",
       "      <td>ba3329d8-5fc2-3b70-bd63-131797230176</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49979</th>\n",
       "      <td>f3d2ed80-f677-3d2b-b06d-d2b627ecb89e</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49980</th>\n",
       "      <td>09d81155-92f8-3a1b-a96c-18a60ddb0af5</td>\n",
       "      <td>4.483332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49981</th>\n",
       "      <td>180e945e-7b9a-3926-a0d5-c5d37adc6397</td>\n",
       "      <td>4.488064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49982</th>\n",
       "      <td>48f19b81-9830-342c-9b56-5d16461aaf59</td>\n",
       "      <td>4.561512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49983</th>\n",
       "      <td>d3e78b57-56f3-3477-8212-6375541c4c52</td>\n",
       "      <td>4.552526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49984</th>\n",
       "      <td>4aebdab0-77b1-3c66-9073-d6b620218825</td>\n",
       "      <td>4.377879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49985</th>\n",
       "      <td>e6eb2ce4-08ec-3574-915c-fd9086a84c74</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49986</th>\n",
       "      <td>79060b6f-647c-35a7-899e-72aba4d1a40d</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49987</th>\n",
       "      <td>e4522edd-00dc-3499-a03c-1e37bdb13934</td>\n",
       "      <td>4.543968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49988</th>\n",
       "      <td>4a9c2aa1-7d33-3c9b-a767-e0deff7b82dd</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49989</th>\n",
       "      <td>00fd7bd9-6dcd-3896-a11a-9f2f2935719e</td>\n",
       "      <td>4.163743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49990</th>\n",
       "      <td>db16ef5b-808a-3156-8651-9fdc9e1df664</td>\n",
       "      <td>4.406581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49991</th>\n",
       "      <td>df17c87e-fb8e-3e30-891f-2becdb4eb335</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49992</th>\n",
       "      <td>4645e289-2bbc-360e-a204-ef3ada0a7d61</td>\n",
       "      <td>3.772744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>23981e76-6173-35b4-8b7b-75f0f15f1c1d</td>\n",
       "      <td>4.478039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>8dec9068-ca74-32c1-92cc-aca4deb574af</td>\n",
       "      <td>4.456880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>cf44f035-ad85-35db-a682-82238d4ad824</td>\n",
       "      <td>3.850903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>694536a0-3b9a-3670-824f-c3bff9d510fc</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>f0f9bc5e-caab-33a9-9c55-c79ad25a4354</td>\n",
       "      <td>4.166520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>fd321d66-7947-31eb-b940-d1751da3d3e3</td>\n",
       "      <td>4.692336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>5a6aa35a-e877-3f9e-8a28-91d801c2c71a</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         Id     Score\n",
       "0      d1c0b520-43c2-3060-843f-711422be08e7  3.089528\n",
       "1      fed809ea-6c05-3cb5-864f-0b10199f38cf  4.419052\n",
       "2      62880c2f-19ad-367d-ba6b-285984fd2e1c  4.084131\n",
       "3      7a35d3c4-f5ff-384e-b0f9-032f3be7d81b  4.546053\n",
       "4      64f0cdd4-26f9-3034-b109-eac59a1f4f30  4.691096\n",
       "5      8fa29e8b-49e4-3d65-8dc1-9402c61dabd9  4.645653\n",
       "6      512fc844-4f4f-3cdc-93a7-2f209937420a  3.875635\n",
       "7      eb68cf16-ae60-3a1e-b648-ad92cc12c9c4  4.045265\n",
       "8      89e8804e-84d1-37ce-8af0-ae3cd38f7b06  4.462513\n",
       "9      6b6abc03-ae74-35a1-9b4f-ec4525ab3b72  4.522406\n",
       "10     a96151dc-0d88-3a37-bd19-3dd426704414  3.894797\n",
       "11     64a2cd30-8e29-3df4-b580-5b5d82103c31  5.000000\n",
       "12     33a3b298-5e75-3b39-b445-d7deafc5704a  4.614712\n",
       "13     26ea0a19-d13b-3ffe-8c60-0f34a4a0a634  4.277895\n",
       "14     81915b19-7cc0-31c1-b271-c005a5d01bb5  4.681723\n",
       "15     82a3142b-edb1-31e0-977f-6cdb96e1dbf7  4.495889\n",
       "16     01cca46c-88cd-3c87-9583-f65999bd35c4  4.636938\n",
       "17     b45685cd-641c-3ed8-bfb9-8ffdb5b29d1b  3.842805\n",
       "18     2b4e3e0b-1f54-3fb7-95e5-e50c638a5435  2.444538\n",
       "19     9e20d005-b766-3b53-ade8-2cbf226bd318  5.000000\n",
       "20     4df2cbb1-6994-3998-8cb6-21e1bd00e041  4.526083\n",
       "21     3539405d-e389-3ec0-923c-ddaeecd3803c  3.891205\n",
       "22     284baf9d-a66e-3717-9735-e7bfe3c90e5b  5.000000\n",
       "23     4ea8f7a1-4219-3731-a960-888b0061c34b  4.500006\n",
       "24     99cea5e2-95d1-3b1e-99c4-5f989a7e498a  5.000000\n",
       "25     5cceae79-a341-375e-863c-23bf1a98a4ee  4.330118\n",
       "26     106b3b7d-4b95-3006-bb3a-24e577c44967  4.547858\n",
       "27     89cd390d-4bde-3421-8806-199f98d74cb1  5.000000\n",
       "28     3e1b59a4-d5b6-30f3-b6ea-8d70b04fe445  4.452993\n",
       "29     5d98909a-41f1-3ec4-9ea1-145d93e8530f  3.861439\n",
       "...                                     ...       ...\n",
       "49970  75e3f467-ee98-3dcc-812f-72f7ab6bf80c  5.000000\n",
       "49971  78039063-aa5d-34ab-a1a6-00d49ff6a16f  5.000000\n",
       "49972  f8aec5d7-b10c-3005-891a-3e353765cea0  3.583248\n",
       "49973  a056a723-7270-3bf3-ba48-b590d2367f33  4.650035\n",
       "49974  fb7dfd18-b884-3bb0-88da-f6a91aa2908e  5.000000\n",
       "49975  61c44262-2ba0-3764-b2fa-034afa11ba63  3.804826\n",
       "49976  168ffccc-97f2-3292-89f1-b5e69e2b0342  4.511363\n",
       "49977  11d09de4-6b44-384e-b2ab-e09c70ccb7d2  4.446552\n",
       "49978  ba3329d8-5fc2-3b70-bd63-131797230176  5.000000\n",
       "49979  f3d2ed80-f677-3d2b-b06d-d2b627ecb89e  5.000000\n",
       "49980  09d81155-92f8-3a1b-a96c-18a60ddb0af5  4.483332\n",
       "49981  180e945e-7b9a-3926-a0d5-c5d37adc6397  4.488064\n",
       "49982  48f19b81-9830-342c-9b56-5d16461aaf59  4.561512\n",
       "49983  d3e78b57-56f3-3477-8212-6375541c4c52  4.552526\n",
       "49984  4aebdab0-77b1-3c66-9073-d6b620218825  4.377879\n",
       "49985  e6eb2ce4-08ec-3574-915c-fd9086a84c74  5.000000\n",
       "49986  79060b6f-647c-35a7-899e-72aba4d1a40d  5.000000\n",
       "49987  e4522edd-00dc-3499-a03c-1e37bdb13934  4.543968\n",
       "49988  4a9c2aa1-7d33-3c9b-a767-e0deff7b82dd  5.000000\n",
       "49989  00fd7bd9-6dcd-3896-a11a-9f2f2935719e  4.163743\n",
       "49990  db16ef5b-808a-3156-8651-9fdc9e1df664  4.406581\n",
       "49991  df17c87e-fb8e-3e30-891f-2becdb4eb335  5.000000\n",
       "49992  4645e289-2bbc-360e-a204-ef3ada0a7d61  3.772744\n",
       "49993  23981e76-6173-35b4-8b7b-75f0f15f1c1d  4.478039\n",
       "49994  8dec9068-ca74-32c1-92cc-aca4deb574af  4.456880\n",
       "49995  cf44f035-ad85-35db-a682-82238d4ad824  3.850903\n",
       "49996  694536a0-3b9a-3670-824f-c3bff9d510fc  5.000000\n",
       "49997  f0f9bc5e-caab-33a9-9c55-c79ad25a4354  4.166520\n",
       "49998  fd321d66-7947-31eb-b940-d1751da3d3e3  4.692336\n",
       "49999  5a6aa35a-e877-3f9e-8a28-91d801c2c71a  5.000000\n",
       "\n",
       "[50000 rows x 2 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
